import os
import gensim
from create_corpus import MyGensim

def get_each_file_from_testset(dir_path='./test'):
    if os.path.isdir(dir_path):
        for _file in os.listdir(dir_path):
            file_path = dir_path + '/' + _file
            f = open(file_path, "r")
            content = f.read()
            f.close()
            yield content
            
class Perplexity:

    def __init__(self):
        self.mygensim = MyGensim()
        self.lda_model = self.mygensim.get_ldamodel()
        
    def calculate_perplexity(self):
        chunk = list()
        for doc in get_each_file_from_testset():
            vec = self.mygensim.doc2bow(doc)
            chunk.append(vec)
        self.lda_model.log_perplexity(chunk)

if __name__ == "__main__":
     perplexity = Perplexity()
     perplexity.calculate_perplexity()
    
    
